# -*- coding:utf-8 -*-
"""
原始数据源进行去重操作，因为发现有些新闻没有提到相应的股票，但是新浪也将它归到了该股票下面。
所以现在讲数据库中的数据进行去重，以后自己切完词之后查找匹配相应的股票实体。
原始数据库共有：2621890条记录
使用url去重

author：yinzm
"""
import pymongo
import redis
mongoClient = pymongo.MongoClient("localhost", 27017)
db = mongoClient.sinafinance
collection = db.sinafinance
redis_db = redis.Redis(host="localhost", port=6379, db=1)
redis_data = "Mongodb_Item"

newCollection = db.sinafinance2

def run():
    cnt = 1
    for item in collection.find():
        print(cnt)
        if redis_db.hexists(redis_data, item['link']) == False:
            # 该记录是新的记录，那么插入redis，newCollection
            redis_db.hset(redis_data, item['link'], 0)
            item.pop('code') # 不需要该字段
            newCollection.insert(item)
        cnt += 1

if __name__ == "__main__":
    run()

# # ok
# # 测试直接插入到sinafinance2中
# newCollection.insert({'id':'yinzm', 'age':24})

# # ok
# # 测试从数据库中抽取数据
# cnt = 0
# for item in collection.find():
#     print(item['link'])
#     cnt += 1
#     if cnt == 3:
#         break

## ok
# # 测试多次使用collection.find()能否获取相同的记录序列
# def getItemFromMongodb(id):
#     mongoClient = pymongo.MongoClient("localhost", 27017)
#     db = mongoClient.sinafinance
#     collection = db.sinafinance
#
#     cnt = 0
#     for item in collection.find():
#         print(item['_id'])
#         cnt += 1
#         if cnt == 3:
#             break
#     mongoClient.close()
#
# if __name__ == "__main__":
#     getItemFromMongodb(1)
#     print("==============")
#     getItemFromMongodb(2)

# # ok
# # 测试将数据插入到redis指定数据库中
# redis_db = redis.Redis(host="localhost", port=6379, db=1)
# redis_data = "Mongodb_Item"
# redis_db.hset(redis_data, collection.find_one()['link'], 0)
# if redis_db.hexists(redis_data, collection.find_one()['link']):
#     print("exists!")

# redis_db = redis.Redis(host="localhost", port=6379, db=1)
# redis_data = "Mongodb_Item"
